# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json,os,logging,scrapy,pymongo
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
from scrapy.conf import settings

class NjuptPipeline(object):
    def __init__(self):
        self.file = open('njupt.txt',mode='wb')
    def process_item(self, item, spider):
        self.file.write(item['news_title'].encode("GBK"))
        self.file.write("\n")
        self.file.write(item['news_date'].encode("GBK"))
        self.file.write("\n")
        self.file.write(item['news_url'].encode("GBK"))
        self.file.write("\n")
        return item

class CrcaPipeline(object):
    def __init__(self):
        self.file = open('crca.txt',mode='wb')
    def process_item(self, item, spider):
        self.file.write(item['url']+":\n")
        self.file.write(item['title']+"---")
        self.file.write(item['book'])
        self.file.write("\n")
        return item

class ShangdunPipeline(object):
    def __init__(self):
        connection = pymongo.MongoClient(
            settings['MONGODB_SERVER'],
            settings['MONGODB_PORT']
        )
        db = connection[settings['MONGODB_DB']]
        self.collection = db[settings['MONGODB_COLLECTION']]
    def process_item(self, item, spider):
        valid = True
        for data in item:
            if not data:
                valid = False
                raise DropItem("Missing {0}!".format(data))
        if valid:
            self.collection.insert(dict(item))
            logging.info("添加成功!")
        return item

class ZmsPipeline(ImagesPipeline):

    def get_media_requests(self, item, info):
        #for image_url in item['image_urls']:
        if item['image_urls']:
            yield scrapy.Request(item['image_urls'])

    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("Item contains no images")
        path = '/home/coolnet/zms/'+image_paths[0]
        logging.info("%s下载成功!,路径:%s",item['title'],path)
        os.rename(path,os.path.dirname(path)+'/'+item['title']+os.path.splitext(path)[-1])
        return item
